package com.feheadline.ml.classification;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.Map;

import com.dtruth.dataflow.dao.HBaseDAO;
import com.dtruth.dataflow.util.StringUtil;

public class DataStat {

	public static void main(String[] args) {
		try {
			HBaseDAO hbaseDao = new HBaseDAO();
			File path = new File("F:/machinelearning/temp");
			FileWriter fw = new FileWriter("F:/machinelearning/temp/stat.txt");
			File[] files = path.listFiles();
			for (File f : files) {
				String aclass = f.getName();
				int aclass_count = 0;
				File[] sfiles = f.listFiles();
				if (sfiles == null || sfiles.length <= 0) {
					System.out.println(aclass + ":" + aclass_count);
					fw.write(aclass + ":" + aclass_count + "\n");
					continue;
				}
				for (File sfile : sfiles) {
					String bclass = sfile.getName();
					int bclass_count = 0;
					int total = 0;
					FileReader fr = new FileReader(sfile);
					BufferedReader br = new BufferedReader(fr);
					String line = null;
					while ((line = br.readLine()) != null) {
						if (line.indexOf("www.emkt.com.cn") > -1) {
							String param = line
									.substring(line.indexOf("?ID=") + 4);
							line = "http://www.emkt.com.cn/article/"
									+ param.substring(0, 3) + "/" + param
									+ ".html";
						}
						String key = StringUtil.getMD5(line);
						Map<String, Object> map = hbaseDao.selectRow("crawled",
								key);
						total++;
						if (map.get("title") == null
								&& map.get("article") == null)
							continue;
						bclass_count++;
						aclass_count++;
					}
					br.close();
					fr.close();
					System.out.println(aclass + "@" + bclass + "[" + total
							+ "," + bclass_count + "]");
					fw.write(aclass + "@" + bclass + "[" + total + ","
							+ bclass_count + "]" + "\n");
				}
				fw.write(aclass + ":" + aclass_count + "\n");
				System.out.println(aclass + ":" + aclass_count);
			}
			fw.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}
}
